#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: caijj
@version: V1.0
@file: zg_sz_new_house_info.py
@time: 2022/02/11
"""
import csv
import multiprocessing
import os
import queue
import random
import re
import threading
import uuid
from time import sleep
import pandas as pd
import requests
# from fake_useragent import UserAgent
from fake_useragent import UserAgent
from lxml import etree
from selenium import webdriver
from sqlalchemy import create_engine
from sqlalchemy.exc import ResourceClosedError
from common.globalvar import GlobalVar
from config.conf import cm
from tools import timeutil, proxy_tool
from tools.DingTalk_Base import DingTalk_Disaster
from tools.database_tool import DataBaseTool
from tools.decorator import update_flag, send_ding_msg, get_ip
from tools.ip_tool import IpTool

mutex_lock = None
estate_info_queue = None
ding = DingTalk_Disaster()
table_root_directory = 'mega_bk_{city}_xf_house_type_info'
file_root_directory = 'mega_bk_{city}_xf_house_type_info.csv'
estate_info_table_root_directory = 'mega_bk_{city}_xf_estate_info'
estate_info_file_root_directory = 'beike_{city}_xf.txt'


class GetModelInfo:

    @staticmethod
    def agent():
        ua = UserAgent(verify_ssl=False)
        headers = {'User-Agent': ua.random,
                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,' \
                             '*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 '}
        return headers

    @staticmethod
    def get_ip_list():
        ip_list = IpTool().get_ip()
        GlobalVar.ip_list = ip_list

    def update_ip_list(self):
        now_ip_list = GlobalVar.ip_list
        if len(now_ip_list) == 1:
            self.get_ip_list()
        else:
            GlobalVar.ip_list = now_ip_list[1:]

    @staticmethod
    def chrome_enter_target_page(url='https://su.xinfang.zhuge.com/', ip=None,
                                 chromedriver_path=r"D:\Anaconda3\chromedriver.exe",):  # chrome进入目标页面
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # proxyauth_plugin_path = proxy_tool.create_proxyauth_extension(ip.split(':')[0], ip.split(':')[1])
        # chrome_options.add_extension(proxyauth_plugin_path)
        # chrome_options.add_argument("--proxy-server=http://{ip}".format(ip=ip))
        web_driver = webdriver.Chrome(options=chrome_options)
        # web_driver = webdriver.Chrome(executable_path=chromedriver_path, options=chrome_options)
        web_driver.maximize_window()
        web_driver.get(url)
        return web_driver

    @staticmethod
    def quit(web_driver):  # 退出
        if web_driver is not None:
            web_driver.quit()
            sleep(5)

    def get_new_house_model(self, create_time, table_name, file_name):
        web_driver = None
        while not estate_info_queue.empty():
            try:
                estate_info = estate_info_queue.get()
                model_info_list = []
                estate_id = estate_info[0]
                resblock_name = estate_info[1]
                detail_link = estate_info[2]
                # web_driver = self.chrome_enter_target_page(GlobalVar.ip_list[0], detail_link)
                web_driver = self.chrome_enter_target_page(detail_link)
                html_source = web_driver.page_source
                if html_source == '<html><head></head><body></body></html>':
                    print('网络异常')
                    self.quit(web_driver)
                    estate_info_queue.put(estate_info)
                elif '未连接到互联网' in html_source:
                    print('未连接到互联网ip已失效' + str(timeutil.dt_strptime("%Y-%m-%d %H%M%S")))
                    self.quit(web_driver)
                    estate_info_queue.put(estate_info)
                    # self.update_ip_list()
                elif '无法访问此网站' in html_source:
                    print('无法访问此网页ip已失效' + str(timeutil.dt_strptime("%Y-%m-%d %H%M%S")))
                    self.quit(web_driver)
                    estate_info_queue.put(estate_info)
                    # self.update_ip_list()
                elif '访问频次过高或周边同事/邻居对贝壳房源存在恶意爬取行为' in html_source:
                    print('对贝壳房源存在恶意爬取行为需手动验证' + str(timeutil.dt_strptime("%Y-%m-%d %H%M%S")))
                    self.quit(web_driver)
                    estate_info_queue.put(estate_info)
                    # self.update_ip_list()
                tree = etree.HTML(html_source)
                model_info_parent_ele_list = tree.xpath("//div[@class='modal-content']//li[@class='item top-item "
                                                        "animation'][1]/ul")
                if len(model_info_parent_ele_list) > 1:  # 当前房源存在户型
                    for model_info_parent_ele in model_info_parent_ele_list:
                        model_id = uuid.uuid4()
                        index = model_info_parent_ele_list.index(model_info_parent_ele)
                        data_id = tree.xpath("//div[@class='modal-content']//li[contains(@class,'item top-item')]["
                                             "1]/ul/@data-id")[index]
                        img_src = tree.xpath(
                            "//div[@class='modal-content']//li[@class='item top-item animation'][1]/ul[" + str(
                                index + 1) + "]/img/@src")[0]
                        house_type_detail = \
                            tree.xpath("//div[@class='modal-content']//li[@class='item top-item animation']["
                                       "1]/ul[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                   "@class='content-title']")[0].text.strip()
                        house_type_status = \
                            tree.xpath("//div[@class='modal-content']//li[@class='item top-item animation']["
                                       "1]/ul[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                   "@class='content-title']/span")[0].text
                        house_area_text = \
                            tree.xpath("//div[@class='modal-content']//li[@class='item top-item animation']["
                                       "1]/ul[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                   "@class='content-area']")[0].text
                        building_area = house_area_text.split(' ')[0] + house_area_text.split(' ')[1]
                        orientation = house_area_text.split(' ')[2][1:-1]
                        price_ele = tree.xpath(
                            "//div[@class='modal-content']//li[@class='item top-item animation'][1]/ul"
                            "[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                   "@class='content-price']/span")
                        if len(price_ele) == 0:
                            price = '价格待定'
                        else:
                            price = \
                                tree.xpath("//div[@class='modal-content']//li[@class='item top-item animation'][1]/ul"
                                           "[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                  "@class='content-price']/span")[0].text
                        house_type = house_type_detail[0:1]
                        model_info_list.append(
                            [model_id, estate_id, data_id, house_type, create_time, house_type_detail, img_src,
                             house_type_status, building_area, orientation, price, resblock_name] + estate_info[3:7])

                else:  # 不存在户型
                    pass
                df = pd.DataFrame(data=model_info_list,
                                  columns=['id', 'resblock_id', 'data_id', 'house_type', 'create_time',
                                           'house_type_detail', 'img_src', 'house_type_status', 'building_area',
                                           'orientation', 'price', 'resblock_name', 'resblock_status', 'resblock_type',
                                           'region', 'district'])
                # 保存为csv格式
                # 申请获取锁，此过程为阻塞等待状态，直到获取锁完毕
                # mutex_lock.acquire()
                df.to_csv(file_name, mode="a",
                          encoding="utf-8",
                          header=False,
                          index=False)
                # 释放锁
                # mutex_lock.release()
                self.quit(web_driver)
            except Exception as e:
                print(e)
                estate_info_queue.put(estate_info)
                self.quit(web_driver)
                # self.update_ip_list()
                print('未知异常 + 户型链接{detail_link} + {time}'.format(detail_link=detail_link,
                                                                 time=str(timeutil.dt_strptime("%Y-%m-%d %H%M%S"))))

    @staticmethod
    def get_house_detail(table_name='mega_bk_sz_xf_estate_info'):  # 获取贝壳新房信息
        sql = "select id,detail_url from {}".format(table_name)
        db = DataBaseTool()
        result = db.select_sql(sql)
        # print(len(result))
        # step = len(result) // process_count
        # # remainder = len(result) % process_count
        # for i in range(1, process_count + 1):
        #     if i == range(1, process_count + 1)[-1]:
        #         data = result[step * (i - 1):]
        #     else:
        #         data = result[step * (i - 1):step * i]
        #     data_list.append(data)
        return result

    @staticmethod
    def get_estate_info_from_txt(csv_name):  # 获取贝壳新房信息
        df = pd.read_csv(csv_name, sep=',', header=None)
        df.columns = [
            'id', 'resblock_name', 'detail_url', 'resblock_status',
            'resblock_type', 'region', 'district', 'position_info', 'house_type',
            'house_size', 'unit_price', 'total_price', 'tag'
        ]
        df.drop_duplicates(subset=['detail_url'], keep='first', inplace=True)
        return df.values.tolist()

    @staticmethod
    @update_flag()
    def insert_new_house_model_data(csv_name, table_name):
        engine = create_engine('mssql+pymssql://base:base@192.168.106.21:1433/JRZF_BASE?charset=utf8')
        # 读取本地CSV文件
        if not os.path.exists(csv_name):
            raise FileNotFoundError("配置文件%s不存在！" % csv_name)
        else:
            sql_result = DataBaseTool().get_column_name_sql(table_name)
            column_name_list = [column_item[0] for column_item in sql_result]
            df = pd.read_csv(csv_name, sep=',', header=None)
            df.columns = column_name_list
            try:
                df.to_sql(table_name, engine, index=False, if_exists='fail')
            except ValueError as e:
                try:
                    pd.read_sql_query("delete from {}".format(table_name), con=engine)
                    # print('进入删除数据逻辑')
                except ResourceClosedError as e:
                    # print('进入插入数据逻辑')
                    df.to_sql(table_name, engine, index=False, if_exists='append')

    @staticmethod
    def check_file_is_exists(csv_name):
        if os.path.isfile(csv_name):
            os.remove(csv_name)

    # 保存户型信息
    # @send_ding_msg()
    # @get_ip()
    def save_house_model_data(self):
        global estate_info_queue, mutex_lock
        city_dict = GlobalVar.city_env
        for city in city_dict.keys():
            if city == 'sz':
                new_city = 'su'
            else:
                new_city = city
            GlobalVar.now_city = city
            time = timeutil.dt_strptime("%Y-%m-%d %H%M%S")
            estate_model_info_table_name = table_root_directory.format(city=city)
            estate_model_info_file_name = file_root_directory.format(city=city)
            estate_info_file_name = estate_info_file_root_directory.format(city=new_city)

            data_list = self.get_estate_info_from_txt(cm.tmp_file_path(estate_info_file_name))  # 获取楼盘信息
            self.check_file_is_exists(cm.tmp_file_path(estate_model_info_file_name))
            # 将所有新房信息数据放入先进先出FIFO队列中
            # 队列的写入和读取都是阻塞的，故在多线程情况下不会乱
            # 创建一个队列
            estate_info_queue = queue.Queue(len(data_list))
            # 写入新房信息数据写到队列
            for i in range(len(data_list)):
                estate_info_queue.put(data_list[i])
            # 创建一个线程锁，防止多线程写入文件时发生错乱
            mutex_lock = threading.Lock()
            # 线程数为50，在一定范围内，线程数越多，速度越快
            for i in range(3):
                t = threading.Thread(target=self.get_new_house_model, name='LoopThread' + str(i),
                                     args=(time, estate_model_info_table_name,
                                           cm.tmp_file_path(estate_model_info_file_name)))
                # t = threading.Thread(target=self.get_house_type_by_api, name='LoopThread' + str(i),
                #                      args=(time, estate_model_info_table_name,
                #                            cm.tmp_file_path(estate_model_info_file_name)))
                t.start()
            # self.get_new_house_model(time, estate_model_info_table_name, cm.tmp_file_path(estate_model_info_file_name))

    def get_house_type_by_api(self, create_time, table_name, file_name):
        while not estate_info_queue.empty():
            try:
                estate_info = estate_info_queue.get()
                model_info_list = []
                estate_id = estate_info[0]
                resblock_name = estate_info[1]
                detail_link = estate_info[2]
                headers = self.agent()  # 构造请求头
                res = requests.get(detail_link, headers=headers)
                # print(res.text)
                tree = etree.HTML(res.text)
                if res.text == '<html><head></head><body></body></html>':
                    print('网络异常')
                    estate_info_queue.put(estate_info)
                elif '访问频次过高或周边同事/邻居对贝壳房源存在恶意爬取行为' in res.text:
                    print('对贝壳房源存在恶意爬取行为需手动验证' + str(timeutil.dt_strptime("%Y-%m-%d %H%M%S")))
                    estate_info_queue.put(estate_info)
                model_info_parent_ele_list = tree.xpath(
                    "//div[@class='modal-content']//li[contains(@class,'item top-item')]["
                    "1]/ul")
                if len(model_info_parent_ele_list) > 1:  # 当前房源存在户型
                    for model_info_parent_ele in model_info_parent_ele_list:
                        model_id = uuid.uuid4()
                        index = model_info_parent_ele_list.index(model_info_parent_ele)
                        img_src = tree.xpath(
                            "//div[@class='modal-content']//li[contains(@class,'item top-item')][1]/ul[" + str(
                                index + 1) + "]/img/@src")[0]
                        data_id = tree.xpath("//div[@class='modal-content']//li[contains(@class,'item top-item')]["
                                             "1]/ul/@data-id")[index]
                        house_type_detail = \
                            tree.xpath("//div[@class='modal-content']//li[contains(@class,'item top-item')]["
                                       "1]/ul[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                   "@class='content-title']")[0].text.strip()
                        house_type_status = \
                            tree.xpath("//div[@class='modal-content']//li[contains(@class,'item top-item')]["
                                       "1]/ul[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                   "@class='content-title']/span")[0].text
                        house_area_text = \
                            tree.xpath("//div[@class='modal-content']//li[contains(@class,'item top-item')]["
                                       "1]/ul[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                   "@class='content-area']")[0].text
                        building_area = house_area_text.split(' ')[0] + house_area_text.split(' ')[1]
                        orientation = house_area_text.split(' ')[2][1:-1]
                        price_ele = tree.xpath(
                            "//div[@class='modal-content']//li[contains(@class,'item top-item')][1]/ul"
                            "[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                   "@class='content-price']/span")
                        if len(price_ele) == 0:
                            price = '价格待定'
                        else:
                            price = \
                                tree.xpath("//div[@class='modal-content']//li[contains(@class,'item top-item')][1]/ul"
                                           "[" + str(index + 1) + "]/div[@class='card-content']/div["
                                                                  "@class='content-price']/span")[0].text
                        house_type = house_type_detail[0:1]
                        model_info_list.append(
                            [model_id, estate_id, data_id, house_type, create_time, house_type_detail, img_src,
                             house_type_status, building_area, orientation, price, resblock_name] + estate_info[3:7])
                else:  # 不存在户型
                    pass
                df = pd.DataFrame(data=model_info_list,
                                  columns=['id', 'resblock_id', 'data_id', 'house_type', 'create_time',
                                           'house_type_detail', 'img_src', 'house_type_status', 'building_area',
                                           'orientation', 'price', 'resblock_name', 'resblock_status',
                                           'resblock_type', 'region', 'district'])
                # 保存为csv格式
                # 申请获取锁，此过程为阻塞等待状态，直到获取锁完毕
                # mutex_lock.acquire()
                df.to_csv(file_name, mode="a",
                          encoding="utf-8",
                          header=False,
                          index=False)
                # 释放锁
                # mutex_lock.release()
                sleep(random.randint(30, 50))
            except Exception as e:
                # print(e)
                estate_info_queue.put(estate_info)
                print('未知异常 + 户型链接{detail_link} + {time}'.format(detail_link=detail_link,
                                                                 time=str(timeutil.dt_strptime("%Y-%m-%d %H%M%S"))))


if __name__ == '__main__':
    GetModelInfo().save_house_model_data()
    # GetModelInfo().get_house_type_by_api()
